package org.fae.util;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.StringTokenizer;
import java.util.Vector;

import org.eclipse.jetty.http.HttpStatus;

import com.gargoylesoftware.htmlunit.ScriptResult;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

// $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
// $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
/**
 * This is the main class for processing/traversing a URL.
 * @author jheckel
 */
public class URLProcessor {

  public static boolean FIND_EVENTS = true;
  public static boolean EVAL_SCRIPT = true;
  public static boolean FIND_LINKS = true;

  static boolean m_processingURL = false;

  private Processor m_processor;

  // ==========================================================================
  // ==========================================================================
  /** Will process/read the given URL. DOES TRAVERSE HERE!
   * Creates an instance of URLProcessor class and then calls the 'process'
   * method. When complete will look at links (HREF) found and normalize
   * the link into a true URL and then traverse it, increasing 'depth'.
   * @throws MalformedURLException 
   */
  public void traverseURL(FaeUtil faeUtil, String urlFrom, String url, int depth, String baseURL)
    throws MalformedURLException {
    // check if should process/traverse given url
    //if (!traverseIt(depth, FaeUtil.m_urlCount, 0, url))
    //return;
    //    // already read this URL?
    //    if (FaeUtil.m_urlsRead.contains(url))
    //      return;
    //
    //    // past depth to search?
    //    if (depth > FaeUtil.MAX_DEPTH)
    //      return;
    //
    //    // still on the proper domain or host?
    //    if (FaeUtil.m_host != null && FaeUtil.m_host.length() > 0) {
    //      if (FaeUtil.m_traverseSubDomains) {
    //        if (!url.contains(FaeUtil.m_host)) {
    //          return;
    //        }
    //      }
    //      else {
    //        if (!url.startsWith(FaeUtil.m_host)) {
    //          return;
    //        }
    //      }
    //    }

    faeUtil.debug("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
    faeUtil.debug(faeUtil.m_urlCount + ": TRAVERSING " + url + " (" + depth + ":" + baseURL + ")");

    for (int x = 1; x < depth; ++x)
      faeUtil.m_treeRepresentation.append("\t");
    faeUtil.m_treeRepresentation.append(url + "\n");
    //FaeUtil.m_treeRepresentation.append(url + " (" + FaeUtil.m_urlCount + ")\n");

    URLProcessor processor = new URLProcessor();
    // process the URL - read page, find events, evaluate script, find links
    processor.process(faeUtil, urlFrom, url);
    int urlNum = processor.getProcessor().getUrlNum();

    // if going to go past DEPTH then don't bother wasting cpu cycles
    if ((depth + 1) <= faeUtil.DEPTH) {
      // get list of links found
      Vector<String> links = processor.getProcessor().getLinksFound();
      faeUtil.debug(urlNum + ": depth:" + depth + ": number links => " + links.size());
      int cnt = 1;
      for (String link : links) {

        // transform the found link into a true URL and get newBase for URL
        String newBase = baseURL;
        if (link.startsWith("http")) {
          faeUtil.debug(urlNum + ":" + cnt + ": depth:" + depth + ": found " + link);
          newBase = link;
        }
        else {
          faeUtil.debug(urlNum + ":" + cnt + ": depth:" + depth + ": found1 " + link);
          if (link.startsWith("/")) {
            if (baseURL.endsWith("/"))
              link = baseURL.toString() + link.substring(1);
            else
              link = baseURL.toString() + link;
          }
          else {
            if (baseURL.endsWith("/"))
              link = baseURL.toString() + link;
            else
              link = baseURL.toString() + "/" + link;
          }
          faeUtil.debug(urlNum + ":" + cnt + ": depth:" + depth + ": found2 " + link);
        }

        // should we traverse this new URL?
        if (traverseIt(faeUtil, url, depth, urlNum, cnt, link))
          traverseURL(faeUtil, url, link, depth + 1, newBase);

        cnt++;
      }
    }

  }

  // ==========================================================================
  // ==========================================================================
  /** Return true if should traverse given url. 
   * @throws MalformedURLException */
  public boolean traverseIt(FaeUtil faeUtil, String urlFrom, int depth, int urlNum, int cnt, String url)
    throws MalformedURLException {
    boolean traverseIt = true;
    String msg = null;

    // check depth
    if (depth > faeUtil.DEPTH) {
      traverseIt = false;
    }

    // check already read
    if (faeUtil.m_urlsRead.contains(url)) {
      traverseIt = false;
      msg = urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " ALREADY READ, NOT PROCESSING";
    }

    // check do not read list
    for (String doNotRead : faeUtil.m_urlsToNotTraverse) {
      if (url.startsWith(doNotRead)) {
        traverseIt = false;
        msg = urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " MATCHES DO NOT READ LIST, NOT PROCESSING";
      }
    }

    if (traverseIt) {

      // check spanDomains
      if (faeUtil.m_ctrl.SPAN_DOMAINS != null) {
        boolean match = false;
        StringTokenizer st = new StringTokenizer(faeUtil.m_ctrl.SPAN_DOMAINS, ",");
        while (st.hasMoreTokens()) {
          String tok = st.nextToken();
          //System.out.println("---------------------------");
          //System.out.println(tok);
          if (tok.equals("*")) {
            match = true;
            break;
          }
          if (url.indexOf(tok + "/") != -1) {
            URL t = new URL(url);
            String prefix = t.getHost().substring(0, t.getHost().indexOf(tok));
            //System.out.println(prefix);
            // count number of periods
            int num = countOccurrences(prefix, '.');
            //System.out.println(num);
            if (num == 0 || num == 1) {
              match = true;
              break;
            }
          }
        }
        //System.out.println(url + ": match => " + match);
        if (!match) {
          traverseIt = false;
          faeUtil.m_filteredURLs.add(url);
          String timing = "\"" + url + "\",\"" + urlFrom + "\"";
          faeUtil.m_filteredURLsCSV.add(timing);
          msg = urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " DOES NOT MATCH ANY SPAN DOMAIN, NOT PROCESSING";
        }
      }
      else {
        traverseIt = false;
        faeUtil.m_filteredURLs.add(url);
        String timing = "\"" + url + "\",\"" + urlFrom + "\"";
        faeUtil.m_filteredURLsCSV.add(timing);
        msg = urlNum + ":" + cnt + ": depth:" + depth + ": " + url + ": NO SPAN DOMAINS GIVEN, NOT PROCESSING";
      }

      //    // check domain or host match
      //    if (FaeUtil.m_host != null && FaeUtil.m_host.length() > 0) {
      //      if (FaeUtil.m_traverseSubDomains) {
      //        if (!url.contains(FaeUtil.m_host)) {
      //          traverseIt = false;
      //          FaeUtil.m_filteredURLs.add(url);
      //          FaeUtil.verbose(urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " DOES NOT MATCH DOMAIN, NOT PROCESSING");
      //        }
      //      }
      //      else {
      //        if (!url.startsWith(FaeUtil.m_host)) {
      //          traverseIt = false;
      //          FaeUtil.m_filteredURLs.add(url);
      //          FaeUtil.verbose(urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " DOES NOT MATCH HOST, NOT PROCESSING");
      //        }
      //      }
      //    }

      // check includeDomains
      //System.out.println(Controller.INCLUDE_DOMAINS);
      if (faeUtil.m_ctrl.INCLUDE_DOMAINS != null) {
        StringTokenizer st = new StringTokenizer(faeUtil.m_ctrl.INCLUDE_DOMAINS, ",");
        while (st.hasMoreTokens()) {
          String tok = st.nextToken();
          //System.out.println(tok);
          if (url.indexOf("/" + tok + "/") != -1) {
            traverseIt = true;
            faeUtil.m_filteredURLs.remove(url);
            String timing = "\"" + url + "\",\"" + urlFrom + "\"";
            faeUtil.m_filteredURLsCSV.remove(timing);
            msg = urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " MATCHES INCLUDE DOMAIN " + tok + ", PROCESSING";
          }
        }
      }

      // check excludeDomains
      //System.out.println(Controller.EXCLUDE_DOMAINS);
      if (faeUtil.m_ctrl.EXCLUDE_DOMAINS != null) {
        StringTokenizer st = new StringTokenizer(faeUtil.m_ctrl.EXCLUDE_DOMAINS, ",");
        while (st.hasMoreTokens()) {
          String tok = st.nextToken();
          //System.out.println(tok);
          if (url.indexOf("/" + tok + "/") != -1) {
            traverseIt = false;
            faeUtil.m_filteredURLs.add(url);
            String timing = "\"" + url + "\",\"" + urlFrom + "\"";
            faeUtil.m_filteredURLsCSV.add(timing);
            msg = urlNum + ":" + cnt + ": depth:" + depth + ": " + url + " MATCHES EXCLUDE DOMAIN " + tok + ", NOT PROCESSING";
          }
        }
      }

    }

    if (msg != null)
      faeUtil.verbose(msg);
    return traverseIt;
  }

  // ==========================================================================
  // ==========================================================================
  /** Will process/read the given URL. DOES NOT TRAVERSE HERE!
   * Creates an instance of Processor class to run on a thread and then 'monitors'
   * how long it takes to process.
   * If takes longer than MAX_WAIT it will kill the thread.
   */
  public void process(FaeUtil faeUtil, String urlFrom, String url) {
    faeUtil.m_urlsRead.add(url);
    int urlNum = faeUtil.m_urlCount;
    faeUtil.m_urlCount++;

    try {
      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      // make a WebClient for each URL
      WebClient webClient = new WebClient(faeUtil.BROWSER_VERSION);
      faeUtil.initWebClient(webClient);
      //webClient.getOptions().setThrowExceptionOnScriptError(false);
      //webClient.getOptions().setUseInsecureSSL(true); // JSH added

      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      // timing variables
      long diff = 0;
      java.util.Date start = new java.util.Date();
      java.util.Date end;

      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      // make the Thread to process with
      m_processor = new Processor(faeUtil, webClient, urlNum, url, urlFrom);
      Thread th = new Thread(m_processor);

      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      // set to processing = true
      m_processingURL = true;

      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      // start processing...check how long it takes and break if too long
      try {
        th.start();
        while (m_processingURL) {
          Thread.sleep(1000);
          end = new java.util.Date();
          diff = end.getTime() - start.getTime();
          //System.out.println(m_processingURL + " " + diff);
          if (diff > faeUtil.WAIT) {
            //System.out.println("*** breaking " + diff);
            faeUtil.m_unprocessedURLs.add(new URL(url));
            String timing = "\"" + url + "\",\"" + urlFrom + "\",";
            timing += m_processor.m_loadTime + ",";
            timing += m_processor.m_findEventsTime + ",";
            timing += m_processor.m_findEventsTime + ",";
            timing += m_processor.m_scriptProcessingTime + ",";
            timing += (m_processor.m_loadTime + m_processor.m_findEventsTime + m_processor.m_findEventsTime + m_processor.m_scriptProcessingTime);
            faeUtil.m_unprocessedURLsCSV.add(timing);
            System.err.println("COULD NOT PROCESS URL: " + url + ". WAITED " + diff + "ms.");
            th.stop();
            Thread.sleep(3000); // give some time for resources to clean up
            break;
          }
        }
      }
      catch (Exception e) {
        e.printStackTrace();
      }
      finally {
        if (webClient != null)
          webClient.closeAllWindows();
      }

      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      webClient = null;

      // ------------------------------------------------------------------------
      // ------------------------------------------------------------------------
      // record timings
      String timing = urlNum + ",\"" + url + "\",\"" + m_processor.m_urlRead + "\",\"" + urlFrom + "\",";
      timing += m_processor.m_statusCode + ",";
      timing += m_processor.m_loadTime + ",";
      timing += m_processor.m_findEventsTime + ",";
      timing += m_processor.m_findEventsTime + ",";
      timing += m_processor.m_scriptProcessingTime + ",";
      timing += (m_processor.m_loadTime + m_processor.m_findEventsTime + m_processor.m_findEventsTime + m_processor.m_scriptProcessingTime);
      faeUtil.m_processedURLsCSV.add(timing);
    }
    catch (Exception e) {
      e.printStackTrace();
    }

  }

  // ==========================================================================
  // ==========================================================================
  /** Will return the Processor instance used in reading/evaluating the URL. */
  public Processor getProcessor() {
    return m_processor;
  }

  // ==========================================================================
  // ==========================================================================
  public static int countOccurrences(String haystack, char needle) {
    int count = 0;
    for (int i = 0; i < haystack.length(); i++) {
      if (haystack.charAt(i) == needle) {
        count++;
      }
    }
    return count;
  }

  // ==========================================================================
  // ==========================================================================
  public static void main(String[] args) {
    try {
      // ======================================================================
      // SETUP FaeUtil
      // ======================================================================
      FaeUtil faeUtil = new FaeUtil();

      faeUtil.VERBOSE = true;

      faeUtil.m_ctrl.EXCLUDE_DOMAINS = "";
      System.out.println("EXCLUDE_DOMAINS => " + faeUtil.m_ctrl.EXCLUDE_DOMAINS);

      faeUtil.m_ctrl.INCLUDE_DOMAINS = "illinois.edu";
      System.out.println("INCLUDE_DOMAINS => " + faeUtil.m_ctrl.INCLUDE_DOMAINS);

      faeUtil.m_ctrl.SPAN_DOMAINS = "";
      //Controller.SPAN_DOMAINS = "illinois.edu,junk.com";
      System.out.println("SPAN_DOMAINS => " + faeUtil.m_ctrl.SPAN_DOMAINS);

      Vector<String> testURLs = new Vector();
      testURLs.add("http://illinois.edu/");
      testURLs.add("http://www.illinois.edu/");
      testURLs.add("http://www.cites.illinois.edu/email/logins.html");
      testURLs.add("http://odos.illinois.edu/newstudent/Programs/WelcomeDays.html");
      testURLs.add("http://illinois.edu/arts/arts.html");
      testURLs.add("http://go.illinois.edu/visioningoutcomesreport");
      testURLs.add("http://www.library.illinois.edu/cmservices");
      testURLs.add("http://www.business.illinois.edu/");
      testURLs.add("http://news.illinois.edu/news/13/0806music_theory_app_HeinrichTaube.html");
      testURLs.add("http://www.uiaa.org/urbana/");
      testURLs.add("http://www.uic.edu/");
      testURLs.add("http://uiedev.extension.uiuc.edu/100yrs/");
      testURLs.add("http://www.online.uillinois.edu/");
      //testURLs.add("xxx");

      // ======================================================================
      // SETUP URLProcessor
      // ======================================================================
      URLProcessor processor = new URLProcessor();

      // ======================================================================
      // TEST
      // ======================================================================
      for (String url : testURLs) {
        System.out.println("=============================================");

        boolean match = false;
        StringTokenizer st = new StringTokenizer(faeUtil.m_ctrl.SPAN_DOMAINS, ",");
        while (st.hasMoreTokens()) {
          String tok = st.nextToken();
          //System.out.println("---------------------------");
          //System.out.println(tok);
          if (url.indexOf(tok + "/") != -1) {
            URL t = new URL(url);
            String prefix = t.getHost().substring(0, t.getHost().indexOf(tok));
            //System.out.println(prefix);
            // count number of periods
            int num = countOccurrences(prefix, '.');
            //System.out.println(num);
            if (num == 0 || num == 1) {
              match = true;
              break;
            }
          }
        }
        System.out.println(url + ": match => " + match);

        boolean traverseIt = processor.traverseIt(faeUtil, "BASE", 0, 0, 0, url);
        System.out.println(url + ": traverseIt => " + traverseIt);
      }

      if (true)
        System.exit(0);

      //String url_str = "http://illinois.edu/";
      //String url_str = "http://www.illinois.edu/";
      //String url_str = "http://business.nd.edu/accountancy/";
      String url_str = "http://bc.edu/";
      //String url_str = "http://ls.wisc.edu/";
      System.out.println("url_str: " + url_str);

      URL url = new URL(url_str);
      System.out.println("url: " + url);

      String baseUrl = url.getProtocol() + "://" + url.getHost() + "/";
      System.out.println("baseUrl: " + baseUrl);

      String domain = url.getHost();
      if (domain.startsWith("www."))
        domain = domain.substring(4);
      System.out.println("domain: " + domain);

      //URI uri = new URI(url_str);
      //FaeUtil.m_host = baseUrl;
      //FaeUtil.m_host = url.getHost();
      //System.out.println("FaeUtil.m_host: " + FaeUtil.m_host);

      // if going to traverse subDomains then set host to domain
      // if not traversing subDomains then do a startsWith check
      // if traversing subDomains then do a contains check
      //      FaeUtil.m_traverseSubDomains = true;
      //      System.out.println("FaeUtil.m_traverseSubDomains: " + FaeUtil.m_traverseSubDomains);
      //      if (FaeUtil.m_traverseSubDomains)
      //        FaeUtil.m_host = domain;
      //      else
      //        FaeUtil.m_host = baseUrl;
      //      System.out.println("FaeUtil.m_host: " + FaeUtil.m_host);

      faeUtil.DEBUG = true;
      System.out.println("FaeUtil.m_debug: " + faeUtil.DEBUG);
      faeUtil.VERBOSE = true;
      System.out.println("FaeUtil.m_verbose: " + faeUtil.VERBOSE);

      FIND_EVENTS = false;
      System.out.println("FIND_EVENTS: " + FIND_EVENTS);
      EVAL_SCRIPT = false;
      System.out.println("EVAL_SCRIPT: " + EVAL_SCRIPT);
      FIND_LINKS = true;
      System.out.println("FIND_LINKS: " + FIND_LINKS);

      faeUtil.DEPTH = 2;
      System.out.println("FaeUtil.DEPTH: " + faeUtil.DEPTH);

      faeUtil.m_urlsToNotTraverse.add("http://msn.com/");
      faeUtil.m_urlsToNotTraverse.add("http://www.bc.edu/offices/pubaf/");
      System.out.println("faeUtil.m_doNotRead: " + faeUtil.m_urlsToNotTraverse);

      faeUtil.m_extensionsToNotProcess.add("pdf");
      faeUtil.m_extensionsToNotProcess.add("xml");
      System.out.println("faeUtil.m_extensionsToNotProcess: " + faeUtil.m_extensionsToNotProcess);

      try {
        faeUtil.m_evaluationScript = FileUtil.concatenateFiles("/home/jheckel/DRESWork/ainspector-util-read-only/test/scripts.txt", true);
      }
      catch (Exception e) {
        e.printStackTrace();
      }
      System.out.println("faeUtil.m_evaluationScript: " + faeUtil.m_evaluationScript);

      URLProcessor tmp = new URLProcessor();

      // ======================================================================
      // ======================================================================
      // IF NOT TRAVERSING
      faeUtil.m_urlCount = 0;
      faeUtil.m_urlsRead.clear(); // need to clear after previous run
      System.out.println("\n\n");
      tmp.process(faeUtil, "BASE", url_str);
      Vector<String> linksFound = tmp.getProcessor().getLinksFound();
      if (linksFound.size() == 0)
        System.out.println("no URLs found");
      int cnt = 0;
      Collections.sort(linksFound);
      for (String found : linksFound) {
        System.out.println(cnt++ + ": " + "found " + found);
      }

      // ======================================================================
      // ======================================================================
      // IF TRAVERSING
      faeUtil.m_urlCount = 0;
      faeUtil.m_urlsRead.clear(); // need to clear after previous run
      System.out.println("\n\n");
      tmp.traverseURL(faeUtil, "BASE", url_str, 1, baseUrl);
      if (faeUtil.m_urlsRead.size() > 1) {
        System.out.println(faeUtil.m_urlsRead.size() + " total URLs traversed. All URLs traversed:");
        System.out.println("IN READ ORDER");
        cnt = 0;
        for (String t : faeUtil.m_urlsRead) {
          System.out.println("\t" + cnt++ + ": " + t);
        }
        System.out.println("IN ALPHABETICAL ORDER");
        cnt = 0;
        Collections.sort(faeUtil.m_urlsRead);
        for (String t : faeUtil.m_urlsRead) {
          System.out.println("\t" + cnt++ + ": " + t);
        }
      }
    }
    catch (Exception e) {
      e.printStackTrace();
    }
  }

  // $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
  // $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
  /** Class that reads the page, find events, evaluate script, and finds links. */
  public static class Processor implements Runnable {

    private FaeUtil m_faeUtil;
    private WebClient m_webClient;
    private int m_urlNum;
    private String m_url;
    private String m_urlFrom;
    public String m_urlRead;
    public int m_statusCode;

    private Vector<String> m_linksFound = new Vector();

    // ==========================================================================
    // ==========================================================================
    public Processor(FaeUtil faeUtil, WebClient webClient, int urlNum, String url, String urlFrom) throws MalformedURLException {
      m_faeUtil = faeUtil;
      m_webClient = webClient;
      m_urlNum = urlNum;
      m_url = url;
      m_urlFrom = urlFrom;
    }

    // ==========================================================================
    // ==========================================================================
    /** Get URL number - mainly used in output display. */
    public int getUrlNum() {
      return m_urlNum;
    }

    // ==========================================================================
    // ==========================================================================
    /** Get URL to process. */
    public String getUrl() {
      return m_url;
    }

    // ==========================================================================
    // ==========================================================================
    /** Gets links found during processing. */
    public Vector<String> getLinksFound() {
      return m_linksFound;
    }

    public long m_loadTime;
    public long m_findLinksTime;
    public long m_findEventsTime;
    public long m_scriptProcessingTime;

    // ==========================================================================
    // ==========================================================================
    @Override
    public void run() {
      m_faeUtil.verbose("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
      System.out.println(m_urlNum + ": PROCESSING " + m_url);
      m_faeUtil.verbose("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");

      // FOR REAL WORK
      try {
        long startTime, endTime;

        // ------------------------------------------------------------------------
        // ------------------------------------------------------------------------
        m_faeUtil.updateStatus("processing", m_url);

        // ------------------------------------------------------------------------
        // ------------------------------------------------------------------------
        // Get the page to analyze
        m_faeUtil.verbose(" ----- READING PAGE");
        //FaeUtil.verbose("----------------------------------------");
        m_webClient.getOptions().setPrintContentOnFailingStatusCode(false);
        m_webClient.getOptions().setThrowExceptionOnScriptError(false);
        m_faeUtil.verbose("\t" + m_urlNum + ": Retrieving DOM... ");
        startTime = System.currentTimeMillis();
        m_webClient.getOptions().setUseInsecureSSL(true); // JSH added
        URL requestUrl = new URL(m_url);
        HtmlPage page = m_webClient.getPage(requestUrl);
        m_faeUtil.debug(" &&&&& " + page.getWebResponse());
        m_faeUtil.debug(" &&&&& " + page.getWebResponse().getStatusCode());
        m_faeUtil.debug(" &&&&& " + page.getWebResponse().getStatusMessage());
        m_faeUtil.debug(" &&&&& " + page.getWebResponse().getWebRequest().getUrl());
        this.m_urlRead = page.getWebResponse().getWebRequest().getUrl().toString();
        this.m_statusCode = page.getWebResponse().getStatusCode();
        //FaeUtil.dumpNode(page);
        endTime = System.currentTimeMillis();
        m_loadTime = endTime - startTime;
        m_faeUtil.verbose("\t" + m_loadTime + " msec.");

        if (!requestUrl.equals(page.getWebResponse().getWebRequest().getUrl())) {
          m_faeUtil.debug(" &&&&& REDIRECTION");
        }

        if (page.getWebResponse().getStatusCode() == HttpStatus.FORBIDDEN_403 || page.getWebResponse().getStatusCode() == HttpStatus.NOT_FOUND_404) {
          try {
            m_faeUtil.m_unprocessedURLs.add(new URL(m_url));
            String timing = "\"" + m_url + "\",\"" + m_urlFrom + "\",";
            timing += m_loadTime + ",";
            timing += m_findEventsTime + ",";
            timing += m_findEventsTime + ",";
            timing += m_scriptProcessingTime + ",";
            timing += (m_loadTime + m_findEventsTime + m_findEventsTime + m_scriptProcessingTime);
            m_faeUtil.m_unprocessedURLsCSV.add(timing);
          }
          catch (MalformedURLException e1) {
            e1.printStackTrace();
          }
        }
        else {
          // ------------------------------------------------------------------------
          // ------------------------------------------------------------------------
          // Find URLs on page
          if (FIND_LINKS) {
            m_faeUtil.verbose(" ----- FINDING LINKS");
            //FaeUtil.verbose("----------------------------------------");
            m_faeUtil.verbose("\t" + m_urlNum + ": Looking for links... ");
            startTime = System.currentTimeMillis();
            m_faeUtil.findLinks(page, "", m_linksFound);
            endTime = System.currentTimeMillis();
            //for (String link : m_linksFound) {
            //FaeUtil.debug("link=> " + link);
            //}
            m_faeUtil.verbose("\t" + m_linksFound.size() + " links found");
            m_findLinksTime = endTime - startTime;
            m_faeUtil.verbose("\t" + m_findLinksTime + " msec.");
          }

          // ------------------------------------------------------------------------
          // ------------------------------------------------------------------------
          // Look for events
          if (FIND_EVENTS) {
            m_faeUtil.verbose(" ----- FINDING EVENTS");
            //FaeUtil.verbose("----------------------------------------");
            m_faeUtil.verbose("\t" + m_urlNum + ": Looking for events... ");
            startTime = System.currentTimeMillis();
            m_faeUtil.findEvents(page, "");
            endTime = System.currentTimeMillis();
            m_findEventsTime = endTime - startTime;
            m_faeUtil.verbose("\t" + m_findEventsTime + " msec.");
          }

          // ------------------------------------------------------------------------
          // ------------------------------------------------------------------------
          // Run the concatenated JavaScript scripts against the page
          if (EVAL_SCRIPT) {
            if (m_faeUtil.m_evaluationScript != null && m_faeUtil.m_evaluationScript.length() > 0) {
              m_faeUtil.verbose(" ----- EVALUATING SCRIPT");
              //FaeUtil.verbose("----------------------------------------");
              m_webClient.getOptions().setThrowExceptionOnScriptError(true);
              m_faeUtil.verbose("\t" + m_urlNum + ": Running evaluation scripts... ");
              startTime = System.currentTimeMillis();
              ScriptResult result = page.executeJavaScript(m_faeUtil.m_evaluationScript);
              m_faeUtil.verbose(result.getJavaScriptResult().toString());
              endTime = System.currentTimeMillis();
              m_scriptProcessingTime = endTime - startTime;
              m_faeUtil.verbose("\t" + m_scriptProcessingTime + " msec.");

              // ------------------------------------------------------------------------
              // ------------------------------------------------------------------------
              // Process the JavaScript result
              long total = m_loadTime + m_findLinksTime + m_findEventsTime + m_scriptProcessingTime;
              m_faeUtil.processResult(m_urlNum, result, total);
            }
          }
        }
      }
      catch (Exception e) {
        e.printStackTrace();
        try {
          m_faeUtil.m_unprocessedURLs.add(new URL(m_url));
          String timing = "\"" + m_url + "\",\"" + m_urlFrom + "\",";
          timing += m_loadTime + ",";
          timing += m_findEventsTime + ",";
          timing += m_findEventsTime + ",";
          timing += m_scriptProcessingTime + ",";
          timing += (m_loadTime + m_findEventsTime + m_findEventsTime + m_scriptProcessingTime);
          m_faeUtil.m_unprocessedURLsCSV.add(timing);
        }
        catch (MalformedURLException e1) {
          e1.printStackTrace();
        }
      }

      // FOR TESTING PURPOSES
      //      try {
      //        System.out.println("hey: " + m_url);
      //        for (int i = 0; i < 10; ++i) {
      //          Thread.sleep(1000);
      //          System.out.println("\t... " + m_url);
      //        }
      //      }
      //      catch (InterruptedException e) {
      //        e.printStackTrace();
      //      }

      m_processingURL = false;

      m_faeUtil.debug(m_urlNum + ": leaving Processor:run");
    }
  }

}
